#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @author Linson 2012
# multidown English articles && radio
from simple_html_dom import *
from threading_pool import WorkerManager
import threading,re,os
# get 51voa.com 's English articles & radio
baseUrl='http://www.51voa.com'
baseFolder='./voa_english/'
th_num=10
th_timeout=60
categories=[]
articles=[]
failed_articles=[]
#init
if not os.path.isdir(baseFolder):
    os.mkdir(baseFolder)
# get categories
categories+=['http://www.51voa.com/VOA_Standard_'+str(i)+'.html' for i in range(1,28)]
#print(categories[410])
workers=WorkerManager(th_num,th_timeout)#test
lockOne=threading.Lock()
i=0
ln=categories.__len__()
def getPage(cl):
    global articles
    global i
    global ln
    try:
        tp=file_get_html(cl)
        tmpList=tp.find("#list li a")
        lockOne.acquire()
        if tmpList.__len__()>0:
            articles+=[j.href for j in tmpList]
        i+=1
        print("Finished ",i,'/',ln)
        lockOne.release()
        tp.__del__()
        del tmpList[:]
    except:
        lockOne.acquire()
        print("Failed :",cl)
        lockOne.release()

for cl in categories[:]:  # TEST
    workers.add_job(getPage,cl)
workers.wait_for_complete()
#print(articles)
del workers
del lockOne
del categories[:]
#get content
workers=WorkerManager(th_num,th_timeout)#test
lockOne=threading.Lock()
i=0
ln=articles.__len__()
def getContent(pl):
    global i
    global ln
    global baseUrl
    try:
        tp=file_get_html(baseUrl+pl)
        ptitle=tp.find("#title ",0).innertext().replace(' ','-')
        pcontent=tp.find("#content ",0).plaintext        
        pcontent=re.sub(r'(<script(\s|\S)*/script>|>)','',pcontent,0,re.M)
        tmpCon="\n".join([c.strip() for c in pcontent.splitlines() if c.strip().__len__()>0])
        lockOne.acquire()
        i+=1
        try:
            with open(baseFolder+ptitle+str(i)+'.txt','w',encoding='utf-8') as f:
                f.write(tmpCon)
                f.close()
        except:
            pass
        print("Finished ",i,'/',ln)
        lockOne.release()
        tp.__del__()
    except:
        lockOne.acquire()
        print("Failed :",pl)
        lockOne.release()
for pl in articles[:]:  # TEST
    workers.add_job(getContent,pl)
workers.wait_for_complete()
#print(articles)
del workers
del lockOne
del articles[:]